from DrissionPage import Chromium
from DrissionPage import SessionPage
from DrissionPage.common import Settings
from datetime import datetime
from dateutil.relativedelta import relativedelta
import time
import re
import traceback

Settings.set_language('zh_cn')  # 设置为中文
file_path = ""
# 检索关键词
keywords = ["动画", "动效"]

class ModuleItem:
    def __init__(self, name, url, monitor_api):
        self.name = name
        self.url = url
        self.monitor_api = monitor_api

# 启动或接管浏览器，并创建标签页对象
main_tab = Chromium().latest_tab
# 这之后出现的弹窗都会自动确认
main_tab.browser.set.auto_handle_alert()
# 跳转到花粉俱乐部
main_tab.get('https://cn.club.vmall.com/mhw/consumer/cn/community/mhwnews/allcirclehome/')

def is_ele_visible(ele):
    style = ele.attr('style')
    if style is not None and re.search(r"display: ?none", style, flags=re.IGNORECASE):
        return False
    return ele.states.is_displayed

# 找到对应的模块
def find_module_by_name(module_container, module_name):
    modules = module_container.children()
    for module in modules:
        text = module.ele("@tag()=div").text
        if text == module_name:
            return module
    return None

def ele_scroll_to_see(ele):
    ele.wait.has_rect()
    ele.scroll.to_see()

def click_ele(ele):
    ele.wait.has_rect()
    ele.click()

def wait_ele_displayed(tab, locator):
    tab.wait.ele_displayed(locator, timeout = 3)
    return tab.ele(locator)

def expand_more_until_tail(parent, locator, more_text):
    while True:
        more_comment_ele = parent.ele(locator)
        if not more_comment_ele:
            print(f'======>>>>>没找到：{more_text}')
        else:
            if not is_ele_visible(more_comment_ele):
                print(f'======>>>>>不可见：{more_text}')
            if not re.search(rf"{more_text}", more_comment_ele.text, flags=re.IGNORECASE):
                print(f'======>>>>>内容不是：{more_text}')
        if not more_comment_ele or not is_ele_visible(more_comment_ele) or not re.search(rf"{more_text}", more_comment_ele.text, flags=re.IGNORECASE):
            print("=====>>>>>expand_more_until_tail：no-more-break")
            break
        try:
            ele_scroll_to_see(more_comment_ele)
            click_ele(more_comment_ele)
            print("=====>>>>>expand_more_until_tail:ckick")
        except Exception:
            print("=====>>>>>expand_more_until_tail:error")
            break

def expand_reply_more_until_tail(reply_comment_list_ele, more_text):
    while True:
        children = reply_comment_list_ele.children()
        more_comment_ele = children[len(children) - 1]
        if not is_ele_visible(more_comment_ele):
            print(f'======>>>>>不可见：{more_text}')
        if not re.search(rf"{more_text}", more_comment_ele.text, flags=re.IGNORECASE):
            print(f'======>>>>>内容不是：{more_text}')
        if not is_ele_visible(more_comment_ele) or not re.search(rf"{more_text}", more_comment_ele.text, flags=re.IGNORECASE):
            print("=====>>>>>expand_reply_more_until_tail-more-break")
            break
        try:
            ele_scroll_to_see(more_comment_ele)
            click_ele(more_comment_ele)
            print("=====>>>>>expand_reply_more_until_tail:ckick")
        except Exception:
            print("=====>>>>>expand_reply_more_until_tail:error")
            break

def process_not_video_card_deail(tab, card_url, card_desc, card_title, is_comment_empty):
    try:
        print("处理非视频卡：")
        print(card_desc)
        print('\n')

        with open(file_path, 'a', encoding='utf-8') as f:
            f.write("标题：\n")
            f.write(card_title + "\n")
            f.write("网址：\n")
            f.write(card_url + "\n")
            f.write("内容描述\n")
            f.writelines(card_desc)
            f.write('\n')

        if is_comment_empty:
            print("没有评论.....\n")
            return True

        comment_container = wait_ele_displayed(tab, '@class:common-list-wrap')
        if not comment_container:
            return False
        # 如果有更多评论，则全部展开
        expand_more_until_tail(comment_container, '@@class=more-common cursor-pointer', '查看更多评论')

        comment_list = comment_container.eles('@tag()=li')
        if not comment_list:
            print("没有评论.....\n")
            with open(file_path, 'a', encoding='utf-8') as f:
                f.write('\n\n\n')
            return True
        for comment_item in comment_list:
            comment_content = comment_item.children()[1]
            more_comments = comment_content.children()[1]
            print('----------------------评论内容：-----------------------')
            print(comment_item.texts())
            print('更多按钮的stayle:')
            print(more_comments.attr('style')) # display: none;
            print('\n')
            with open(file_path, 'a', encoding='utf-8') as f:
                f.write('----------------------评论内容：-----------------------\n')
                f.writelines(comment_item.texts())
                f.write("\n")
            # 如果有更多评论，则需要展开更多评论，然后记录
            if is_ele_visible(more_comments):
                print("还有更多评论，点击展开...\n")
                more_comments.click()
                comment_dialog = wait_ele_displayed(tab, '@class=ui-dialog-v5 ui-dialog-v5-cont')
                if comment_dialog and is_ele_visible(comment_dialog):
                    reply_comment_list_ele = comment_dialog.ele('@class=sub-comment-wrap')
                    # 如果有更多评论回复，则全部展开
                    expand_reply_more_until_tail(reply_comment_list_ele, '查看更多评论')
                    with open(file_path, 'a', encoding='utf-8') as f:
                        f.writelines(reply_comment_list_ele.texts())
                        f.write("\n")
                    print('----------------------更多评论内容：-----------------------')
                    print(reply_comment_list_ele.texts())
                    comment_dialog.ele('@class=ui-dialog-close').click()
                    continue
            # 如果没有更多评论，只记录这一条评论
            print("没有更多评论，不需要展开...\n")
        with open(file_path, 'a', encoding='utf-8') as f:
            f.write('\n\n\n')
        return True
    except Exception:
        traceback.print_exc()
        return False

def process_video_card_detail(tab, card_url, card_desc, card_title, is_comment_empty):
    try:
        print("处理视频卡... \n")
        print(card_desc)
        print('\n')

        with open(file_path, 'a', encoding='utf-8') as f:
            f.write("标题：\n")
            f.write(card_title + "\n")
            f.write("网址：\n")
            f.write(card_url + "\n")
            f.write("内容描述：\n")
            f.writelines(card_desc)
            f.write('\n')

        if is_comment_empty:
            print("没有评论.....\n")
            return True 

        icon_list_ele = wait_ele_displayed(tab, '@class=comment-icon-wrap vertical')
        # 点击拉出评论列表
        if icon_list_ele:
            icon_list_ele.children()[1].click()
        else:
            return False
        comment_container = wait_ele_displayed(tab, '@class:common-list-wrap')
        # 如果有更多评论，则全部展开
        expand_more_until_tail(comment_container, '@@class=more-common cursor-pointer', '查看更多评论')

        comment_list = comment_container.eles('@tag()=li')
        if not comment_list:
            print("没有评论.....\n")
            with open(file_path, 'a', encoding='utf-8') as f:
                f.write('\n\n\n')
            return True
        for comment_item in comment_list:
            ele_scroll_to_see(comment_item)
            comment_content = comment_item.children()[1]
            more_comments = comment_content.children()[1]
            print('----------------------评论内容：-----------------------')
            print(comment_item.texts())
            print('更多按钮的stayle:')
            print(more_comments.attr('style')) # display: none;
            print('\n')
            with open(file_path, 'a', encoding='utf-8') as f:
                f.write('----------------------评论内容：-----------------------\n')
                f.writelines(comment_item.texts())
                f.write("\n")
            # 如果有更多评论，则需要展开更多评论，然后记录
            if is_ele_visible(more_comments):
                print("还有更多评论，点击展开...\n")
                more_comments.click()
                comment_dialog = wait_ele_displayed(tab, '@class=comment-detail')
                if comment_dialog and is_ele_visible(comment_dialog):
                    reply_comment_list_ele = comment_dialog.ele('@class=sub-comment-wrap')
                    # 如果有更多评论回复，则全部展开
                    expand_reply_more_until_tail(reply_comment_list_ele, '查看更多评论')
                    with open(file_path, 'a', encoding='utf-8') as f:
                        f.writelines(reply_comment_list_ele.texts())
                        f.write("\n")
                    print('----------------------更多评论内容：-----------------------')
                    print(reply_comment_list_ele.texts())
                    comment_dialog.ele('@class=img-box').click()
                    continue
            # 如果没有更多评论，只记录这一条评论
            print("没有更多评论，不需要展开...\n")
        with open(file_path, 'a', encoding='utf-8') as f:
            f.write('\n\n\n')
        return True
    except Exception:
        traceback.print_exc()
        return False

def process_card_deail(tab, card_url, is_comment_empty, is_not_video, card_desc, card_title):
    detail_title = wait_ele_displayed(tab, '@@class=nav-title@@text()=帖子详情')
    try:
        if not detail_title:
            print("不在帖子详情页... \n")
            if wait_ele_displayed(tab, '@@class=nav-title@@text()=个人主页'):
                print('意外跳转到个人主页，直接返回... \n')
            return
        # 非视频卡
        if is_not_video:
            # # 左侧内容区
            # left_content = wait_ele_displayed(tab, '@class=left-cont')
            # #右侧评论区
            # right_content = wait_ele_displayed(tab, '@class=right-cont')
            processed = process_not_video_card_deail(tab, card_url, card_desc, card_title, is_comment_empty)
            if not processed:
                process_video_card_detail(tab, card_url, card_desc, card_title, is_comment_empty)
        else:
            # video_card_detail = wait_ele_displayed(tab, '@@tag()=video@@class=in-video')
            processed = process_video_card_detail(tab, card_url, card_desc, card_title, is_comment_empty)
            if not processed:
                process_not_video_card_deail(tab, card_url, card_desc, card_title, is_comment_empty)
    finally:
        if detail_title:
            time.sleep(1)
            tab.ele('@class=nav-back').click()

def process_card(card_url, card_title):
    card_detail_tab = Chromium().new_tab('', new_window=False)
    card_detail_tab.listen.start('https://sgw-cn.c.huawei.com/forward/club/content_h5/queryThreadDetail')
    card_detail_tab.get(card_url)
    card_detail_tab.wait.load_start()
    try:
        body = card_detail_tab.listen.wait(timeout=10).response.body # 等待并获取一个数据包
        card_desc = '未获取到内容描述'
        if not body or not body['data']:
            is_comment_empty =  True
            is_not_video = True
            print('======>>>未获取到响应数据body:')
            print(body)
            print('\n')
        else:
            is_comment_empty = not body['data']['postList']
            is_not_video = not body['data']['video']
            content = body['data']['content']
            if content:
                card_desc = content
        process_card_deail(card_detail_tab, card_url, is_comment_empty, is_not_video, card_desc, card_title)
    finally:
        card_detail_tab.close()

def get_dateline_timestamp():
    # 获取当前时间（带时区可替换为 datetime.now(timezone.utc)）
    now = datetime.now()
    # 计算 6 个月前的时间
    six_months_ago = now - relativedelta(months=6)
    # 转换为时间戳（毫秒）
    timestamp = int(six_months_ago.timestamp()) * 1000
    print("6个月前时间戳：" + str(timestamp))
    return timestamp

# 滚动卡片之前如果如果每个容器的卡片数量超过limit_size，多余的卡片会被删除
def scroll_card_list_to_bottom(tab, max_card_num, min_card_num):
    try:
        wait_ele_displayed(tab, '@class=column-cont')
        card_container_list = tab.eles('@class=column-cont')
        max_card_num = max_card_num if max_card_num > min_card_num else min_card_num
        retain_card_num_per_container = int(min_card_num / len(card_container_list))
        current_total_card_num = 0
        for container_index in range(len(card_container_list)):
            card_list = card_container_list[container_index].children()
            current_total_card_num = current_total_card_num + len(card_list)
        if current_total_card_num > max_card_num:
            print(f'=============================>>>>：移除{current_total_card_num - min_card_num}卡片，只保留{min_card_num}条展示')
            for container_index in range(len(card_container_list)):
                card_list = card_container_list[container_index].children()
                # 只保留50条展示
                for index in range(len(card_list) - retain_card_num_per_container):
                    card_list[index].run_js('this.remove()')
        # 等待每一个容器的最后一张卡渲染出来
        for card_container in card_container_list:
            try_count = 1
            while True:
                card_list = card_container.children()
                last_card_index = len(card_list) - 1
                card_list[last_card_index].scroll.to_see()
                time.sleep(1)
                # 等待页面渲染完成
                card = card_container.ele('xpath:./div[position()=' + str(last_card_index + 1) +']')
                if not card:
                    if try_count <= 3:
                        print('未找到卡片：' + str(last_card_index + 1) + '，尝试滚动浏览器到底部，尝试次数：' + str(try_count) + '\n')
                        try_count = try_count + 1
                        time.sleep(1)
                        continue
                else:
                    break # 如果当前卡片已渲染完成，继续跳出循环，继续等待下一张卡片渲染
        # 所有卡都渲染之后滚动页面到底部
        wait_ele_displayed(tab, '@tag()=html').scroll.to_bottom()
    except Exception:
        traceback.print_exc()
    return False

def wait_card_list_data(tab):
    body = tab.listen.wait(timeout = 10).response.body # 等待并获取一个数据包
    if not body['data'] or body['errcode'] != '0':
        print('xxxxxxxxxx：未获取到卡片列表数据！')
    else:
        card_data_list = body['data']['threadBeanList']
        if not card_data_list:
            print('xxxxxxxxxx：卡片列表为空！')
        else: 
            # 只获取最近半年的数据
            last_card_update_time = card_data_list[0]['updateTime']
            if last_card_update_time < get_dateline_timestamp():
                print('xxxxxxxxxx：半年之前的数据！')
            else:
                return card_data_list
    return []

def process_module_item(tab, module_item, module_item_data):
    module_item.click()
    # main_tab.get(module_item.url)
    tab.wait.load_start()
    wait_ele_displayed(tab, '@id=vueWaterfall')
    tab.listen.start(module_item_data.monitor_api)
    wait_ele_displayed(tab, '@text()=最新').click()
    tab.wait.load_start()
    while True:
        try:
            card_data_list = wait_card_list_data(tab)
            print('==========>>获取到卡片数据：' + str(len(card_data_list)) + " 条")
            if not card_data_list:
                break
            if scroll_card_list_to_bottom(tab, 1500, 100):
                break
            for card_data in card_data_list:
                try:
                    card_title = "" if not card_data['title'] else card_data['title']
                    card_desc = "" if not card_data['description'] else card_data['description']
                    if not any(keyword in card_desc for keyword in keywords) and not any(keyword in card_title for keyword in keywords):
                        print("标题：" + card_title)
                        print("内容描述：" + card_desc)
                        print('不包含检索关键词，跳过...\n')
                        continue
                    if not re.search(r"next", card_desc, flags=re.IGNORECASE) and not re.search(r"next", card_title, flags=re.IGNORECASE):
                        print("标题：" + card_title)
                        print("内容描述：" + card_desc)
                        print('不包含next，跳过...\n')
                        continue
                    if not card_data['videoInfos']:
                        card_url = f'https://cn.club.vmall.com/mhw/consumer/cn/community/mhwnews/question/id_{card_data["threadId"]}'
                    else:
                        card_url = f'https://cn.club.vmall.com/mhw/consumer/cn/community/mhwnews/video/id_{card_data["threadId"]}'
                    with open(file_path, 'a', encoding='utf-8') as f:
                        f.write('标题：\n')
                        f.write(card_title)
                        f.write('网址：\n')
                        f.write(card_url)
                        f.write('\n\n\n')
                    # process_card(card_url, card_title) # 抓取评论内容
                except Exception:
                    traceback.print_exc()
        except Exception:
            traceback.print_exc()
            break
    tab.listen.stop()

# 找到分类内容父容器
module_name_list = ["推荐", "手机", "穿戴", "PC平板", "智慧家", "车生活", "兴趣", "同城", "高校", "其他"]
for module_name in module_name_list:
    file_path = f"D:\workspace\crawler\data\花粉俱乐部-{module_name}.txt"
    module_item_position = 0
    while True:
        try:
            module_container = wait_ele_displayed(main_tab, '@@tag()=div@@class=circle-index-card')
            module = find_module_by_name(module_container, module_name)
            if not module:
                print("xxxxxxx:没有找到模块：" + module_name)
                break
            module.scroll.to_see()
            module_item_list = module.children()[1:]
            if module_item_position >= len(module_item_list):
                break
            process_module_item(main_tab, module_item_list[module_item_position], ModuleItem(
                '',
                '',
                'https://sgw-cn.c.huawei.com/forward/club/content_h5/allPost/3'
            ))
        except Exception:
            print(f'xxxxxxx:{module_name}-模块处理异常：')
            traceback.print_exc()
        finally:
            module_item_position = module_item_position + 1
            main_tab.get('https://cn.club.vmall.com/mhw/consumer/cn/community/mhwnews/allcirclehome/')
            main_tab.wait.load_start()

